{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Segement=4 20折 去掉['end_date', 'start_date'] 分层Kfold\n",
    "#线下0.9455841187427304  #拖网1236 #围网516 #刺网248\n",
    "#线上0.88418\n",
    "#Segement=4 20折 去掉['end_date', 'start_date', 'start_hour', 'end_hour', 'work_days', 'work_seconds'] 分层Kfold\n",
    "#线下0.9457698378018721   #拖网1237 #围网518 #刺网245\n",
    "\n",
    "#Segement=4 5折 去掉['end_date', 'start_date'] 分层Kfold\n",
    "#线下0.940494141280405   #拖网1230 #围网519 #刺网251\n",
    "#Segement=4 5折 去掉['end_date', 'start_date', 'start_hour', 'end_hour', 'work_days', 'work_seconds'] 分层Kfold\n",
    "0.9403440104148132 #拖网1233 #围网512 #刺网255 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Demo4\n",
    "filename = \"demo4\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from datetime import datetime\n",
    "#显示所有列\n",
    "pd.set_option('display.max_columns', None)\n",
    "#显示所有行\n",
    "pd.set_option('display.max_rows', None)\n",
    "\n",
    "train_list = os.listdir('../hy_round1_train_20200102/')\n",
    "test_list = os.listdir('../hy_round1_testA_20200102/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def distance(m, n):\n",
    "    \"\"\"calculate Euclidean Distance\"\"\"\n",
    "    return np.sqrt(np.sum((m - n) ** 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_feature_base(demo):\n",
    "    demo.rename(columns={'渔船ID': \"ID\", \"速度\": \"speed\", \"方向\": \"direction\"}, inplace=True)\n",
    "    demo_train = pd.DataFrame()\n",
    "    \n",
    "    #分割time特征得到day, hour, quarter\n",
    "    tmp = pd.DataFrame()\n",
    "    tmp['time'] = pd.to_datetime(demo['time'],format='%m%d %H:%M:%S')\n",
    "    demo[\"month\"] = tmp[\"time\"].dt.month\n",
    "    demo[\"day\"] = tmp[\"time\"].dt.day\n",
    "    demo[\"hour\"] = tmp[\"time\"].dt.hour\n",
    "    del tmp\n",
    "\n",
    "    #按时间排序\n",
    "    demo.sort_values([\"time\"],inplace=True, ascending=False)\n",
    "\n",
    "    #计算作业持续时间\n",
    "    start = demo.iloc[-1]['time']\n",
    "    end = demo.iloc[0]['time']\n",
    "    diff = datetime.strptime(str(end),\"%m%d %H:%M:%S\") - datetime.strptime(str(start),\"%m%d %H:%M:%S\")\n",
    "\n",
    "    #构建时间起始日,小时\n",
    "    demo_train['ID'] = [demo['ID'][0]]\n",
    "    demo_train['start_date'] = int(start[2:4])\n",
    "    demo_train['start_hour'] = int(start[5:7])\n",
    "    demo_train['end_date'] = int(end[2:4])\n",
    "    demo_train['end_hour'] = int(end[5:7])\n",
    "    demo_train['work_days'] = diff.days\n",
    "    demo_train['work_seconds'] = diff.seconds\n",
    "    \n",
    "    #unique, mean, std, var, min, quantile0.25, median, quantile0.75, max, mode特征: 方向, 速度, x, y\n",
    "    for s in ['x', 'y', 'speed', 'direction']:\n",
    "        temp = demo.groupby('ID')[s].agg({\n",
    "                                          'nunique_' + s: 'nunique', \n",
    "                                          'mean_' + s: 'mean', \n",
    "                                          'std_' + s: 'std', \n",
    "                                          'var_' + s: 'var',\n",
    "                                          'skew_' + s: 'skew',\n",
    "                                          'min_' + s: 'min',\n",
    "                                          'quantile0.25_' + s: lambda x: x.quantile(0.25),\n",
    "                                          'median_' + s: 'median',\n",
    "                                          'quantile0.75_' + s: lambda x: x.quantile(0.75),\n",
    "                                          'max_' + s: 'max',\n",
    "                                          'mode_' + s: lambda x: np.mean(pd.Series.mode(x))}).reset_index()\n",
    "        demo_train = pd.merge(demo_train,temp, on='ID',how='left')\n",
    "\n",
    "    #构建x,y坐标交互特征\n",
    "    demo_train['x_max-min'] = demo_train['max_x'] - demo_train['min_x']\n",
    "    demo_train['y_max-min'] = demo_train['max_y'] - demo_train['min_y']\n",
    "    demo_train['rec_area'] = demo_train['y_max-min'] * demo_train['x_max-min']\n",
    "    demo_train['slope'] = demo_train['y_max-min'] / np.where(demo_train['x_max-min']==0, 0.001, demo_train['x_max-min'])\n",
    " \n",
    "    #活动半径\n",
    "    min_point = np.array([demo_train['min_x'], demo_train['min_y']])\n",
    "    center_point = np.array([demo_train['median_x'], demo_train['median_y']])\n",
    "    max_point = np.array([demo_train['max_x'], demo_train['max_y']])\n",
    "    demo_train['short_r'] = distance(min_point, center_point)\n",
    "    demo_train['long_r'] = distance(max_point, center_point)\n",
    "    \n",
    "       \n",
    "    #缺失值个数比例\n",
    "    demo_train['direction_miss_rate'] = (demo['direction']==0).sum() / len(demo)\n",
    "    demo_train['speed_miss_rate'] = (demo['speed']==0).sum() / len(demo)\n",
    "    demo_train['direct&speed_miss_rate'] = len(demo[demo.direction==0][demo.speed==0]) / len(demo)\n",
    "    \n",
    "    return demo_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SplitDemo_process(demo, Segmentation=7):\n",
    "    demo_train = pd.DataFrame()\n",
    "    #每份的行数\n",
    "    lines = demo.shape[0] // Segmentation\n",
    "    \n",
    "    for i in range(Segmentation):\n",
    "        demo_split = demo.loc[lines*i: lines*(i+1)].reset_index()\n",
    "        demo_train_split = create_feature_base(demo_split)\n",
    "        demo_train = demo_train.append(demo_train_split)\n",
    "    return demo_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7000/7000 [26:00<00:00,  4.28it/s]\n"
     ]
    }
   ],
   "source": [
    "#构建训练集特征\n",
    "train = pd.DataFrame()\n",
    "for file in tqdm(train_list):\n",
    "    demo = pd.read_csv('../hy_round1_train_20200102/' + file)\n",
    "    demo_train = SplitDemo_process(demo, Segmentation=4)\n",
    "    demo_train['type'] = demo['type']\n",
    "    train = train.append(demo_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2000/2000 [07:19<00:00,  4.56it/s]\n"
     ]
    }
   ],
   "source": [
    "#构建测试集\n",
    "test = pd.DataFrame()\n",
    "for file in tqdm(test_list):\n",
    "    demo = pd.read_csv('../hy_round1_testA_20200102/' + file)\n",
    "    demo_test = SplitDemo_process(demo, Segmentation=4)\n",
    "    test = test.append(demo_test)\n",
    "test['type'] = '测试'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#合并数据集合\n",
    "data = train.append(test).reset_index(drop=True)\n",
    "data.to_csv('../input/data_'+'demo4_seg7'+'.csv', index=False)\n",
    "# data = pd.read_csv('../input/data_'+filename+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "type\n",
       "刺网    1018\n",
       "围网    1621\n",
       "拖网    4361\n",
       "测试    2000\n",
       "Name: ID, dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据分布\n",
    "data.groupby('type')['ID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "work_days\n",
       "0    9000\n",
       "1     130\n",
       "2      10\n",
       "Name: ID, dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据分布\n",
    "data.groupby('work_days')['ID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mem. usage decreased to  5.84 Mb (65.2% reduction)\n"
     ]
    }
   ],
   "source": [
    "#降低内存使用\n",
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024**2    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)    \n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return df\n",
    "\n",
    "data = reduce_mem_usage(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>start_date</th>\n",
       "      <th>start_hour</th>\n",
       "      <th>end_date</th>\n",
       "      <th>end_hour</th>\n",
       "      <th>work_days</th>\n",
       "      <th>work_seconds</th>\n",
       "      <th>nunique_x</th>\n",
       "      <th>mean_x</th>\n",
       "      <th>std_x</th>\n",
       "      <th>var_x</th>\n",
       "      <th>skew_x</th>\n",
       "      <th>min_x</th>\n",
       "      <th>quantile0.25_x</th>\n",
       "      <th>median_x</th>\n",
       "      <th>quantile0.75_x</th>\n",
       "      <th>max_x</th>\n",
       "      <th>mode_x</th>\n",
       "      <th>nunique_y</th>\n",
       "      <th>mean_y</th>\n",
       "      <th>std_y</th>\n",
       "      <th>var_y</th>\n",
       "      <th>skew_y</th>\n",
       "      <th>min_y</th>\n",
       "      <th>quantile0.25_y</th>\n",
       "      <th>median_y</th>\n",
       "      <th>quantile0.75_y</th>\n",
       "      <th>max_y</th>\n",
       "      <th>mode_y</th>\n",
       "      <th>nunique_speed</th>\n",
       "      <th>mean_speed</th>\n",
       "      <th>std_speed</th>\n",
       "      <th>var_speed</th>\n",
       "      <th>skew_speed</th>\n",
       "      <th>min_speed</th>\n",
       "      <th>quantile0.25_speed</th>\n",
       "      <th>median_speed</th>\n",
       "      <th>quantile0.75_speed</th>\n",
       "      <th>max_speed</th>\n",
       "      <th>mode_speed</th>\n",
       "      <th>nunique_direction</th>\n",
       "      <th>mean_direction</th>\n",
       "      <th>std_direction</th>\n",
       "      <th>var_direction</th>\n",
       "      <th>skew_direction</th>\n",
       "      <th>min_direction</th>\n",
       "      <th>quantile0.25_direction</th>\n",
       "      <th>median_direction</th>\n",
       "      <th>quantile0.75_direction</th>\n",
       "      <th>max_direction</th>\n",
       "      <th>mode_direction</th>\n",
       "      <th>x_max-min</th>\n",
       "      <th>y_max-min</th>\n",
       "      <th>rec_area</th>\n",
       "      <th>slope</th>\n",
       "      <th>short_r</th>\n",
       "      <th>long_r</th>\n",
       "      <th>direction_miss_rate</th>\n",
       "      <th>speed_miss_rate</th>\n",
       "      <th>direct&amp;speed_miss_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>36000.00000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.00000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.00</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.00000</td>\n",
       "      <td>36000.0000</td>\n",
       "      <td>36000.0</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.0</td>\n",
       "      <td>36000.0</td>\n",
       "      <td>36000.0000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.0</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>3.600000e+04</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "      <td>36000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4499.50000</td>\n",
       "      <td>14.740000</td>\n",
       "      <td>9.691444</td>\n",
       "      <td>14.356194</td>\n",
       "      <td>12.779500</td>\n",
       "      <td>0.004417</td>\n",
       "      <td>62388.20900</td>\n",
       "      <td>43.585139</td>\n",
       "      <td>6.276861e+06</td>\n",
       "      <td>6881.452148</td>\n",
       "      <td>1.955948e+08</td>\n",
       "      <td>0.157837</td>\n",
       "      <td>6.265251e+06</td>\n",
       "      <td>6.271536e+06</td>\n",
       "      <td>6.276682e+06</td>\n",
       "      <td>6281997.50</td>\n",
       "      <td>6.288414e+06</td>\n",
       "      <td>6274504.000</td>\n",
       "      <td>43.585389</td>\n",
       "      <td>5.271188e+06</td>\n",
       "      <td>6120.426270</td>\n",
       "      <td>1.724055e+08</td>\n",
       "      <td>-0.187134</td>\n",
       "      <td>5.260275e+06</td>\n",
       "      <td>5.266671e+06</td>\n",
       "      <td>5.271421e+06</td>\n",
       "      <td>5.275844e+06</td>\n",
       "      <td>5.280934e+06</td>\n",
       "      <td>5.273068e+06</td>\n",
       "      <td>22.717139</td>\n",
       "      <td>inf</td>\n",
       "      <td>1.344727</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>0.147095</td>\n",
       "      <td>0.990234</td>\n",
       "      <td>1.559570</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>1.490234</td>\n",
       "      <td>45.742028</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>0.718262</td>\n",
       "      <td>4.291222</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>316.098194</td>\n",
       "      <td>inf</td>\n",
       "      <td>2.301732e+04</td>\n",
       "      <td>2.066825e+04</td>\n",
       "      <td>2.118399e+09</td>\n",
       "      <td>inf</td>\n",
       "      <td>1.791151e+04</td>\n",
       "      <td>1.693063e+04</td>\n",
       "      <td>0.311768</td>\n",
       "      <td>0.196655</td>\n",
       "      <td>0.139160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2598.11228</td>\n",
       "      <td>8.807784</td>\n",
       "      <td>6.798751</td>\n",
       "      <td>8.605014</td>\n",
       "      <td>7.205213</td>\n",
       "      <td>0.070377</td>\n",
       "      <td>7593.57723</td>\n",
       "      <td>43.507517</td>\n",
       "      <td>2.686800e+05</td>\n",
       "      <td>12175.655273</td>\n",
       "      <td>2.279927e+09</td>\n",
       "      <td>3.158203</td>\n",
       "      <td>2.690936e+05</td>\n",
       "      <td>2.685254e+05</td>\n",
       "      <td>2.689069e+05</td>\n",
       "      <td>269354.75</td>\n",
       "      <td>2.696467e+05</td>\n",
       "      <td>268801.125</td>\n",
       "      <td>43.507534</td>\n",
       "      <td>2.548936e+05</td>\n",
       "      <td>11616.648438</td>\n",
       "      <td>3.866452e+09</td>\n",
       "      <td>3.271484</td>\n",
       "      <td>2.554044e+05</td>\n",
       "      <td>2.547385e+05</td>\n",
       "      <td>2.550029e+05</td>\n",
       "      <td>2.554104e+05</td>\n",
       "      <td>2.570212e+05</td>\n",
       "      <td>2.542882e+05</td>\n",
       "      <td>17.311172</td>\n",
       "      <td>1.762695e+00</td>\n",
       "      <td>1.200195</td>\n",
       "      <td>inf</td>\n",
       "      <td>2.636719e+00</td>\n",
       "      <td>0.448975</td>\n",
       "      <td>1.388672</td>\n",
       "      <td>1.858398</td>\n",
       "      <td>2.539062e+00</td>\n",
       "      <td>5.179688e+00</td>\n",
       "      <td>2.300781</td>\n",
       "      <td>22.710759</td>\n",
       "      <td>61.09375</td>\n",
       "      <td>36.0000</td>\n",
       "      <td>inf</td>\n",
       "      <td>1.740234</td>\n",
       "      <td>27.880334</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>90.8125</td>\n",
       "      <td>98.321777</td>\n",
       "      <td>inf</td>\n",
       "      <td>4.657031e+04</td>\n",
       "      <td>4.740188e+04</td>\n",
       "      <td>4.797043e+10</td>\n",
       "      <td>inf</td>\n",
       "      <td>5.023682e+04</td>\n",
       "      <td>3.199636e+04</td>\n",
       "      <td>0.321533</td>\n",
       "      <td>0.228027</td>\n",
       "      <td>0.225464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>90.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.135336e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-16.062500</td>\n",
       "      <td>5.000250e+06</td>\n",
       "      <td>5.127822e+06</td>\n",
       "      <td>5.130528e+06</td>\n",
       "      <td>5137676.00</td>\n",
       "      <td>5.154938e+06</td>\n",
       "      <td>5144128.000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.502637e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-28.765625</td>\n",
       "      <td>3.345433e+06</td>\n",
       "      <td>4.469721e+06</td>\n",
       "      <td>4.496677e+06</td>\n",
       "      <td>4.512800e+06</td>\n",
       "      <td>4.543204e+06</td>\n",
       "      <td>4.505082e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-7.964844</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2249.75000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>61262.75000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.118429e+06</td>\n",
       "      <td>23.511981</td>\n",
       "      <td>5.528132e+02</td>\n",
       "      <td>-0.591309</td>\n",
       "      <td>6.115724e+06</td>\n",
       "      <td>6.117350e+06</td>\n",
       "      <td>6.118354e+06</td>\n",
       "      <td>6124759.25</td>\n",
       "      <td>6.134904e+06</td>\n",
       "      <td>6118252.000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>5.114772e+06</td>\n",
       "      <td>19.668538</td>\n",
       "      <td>3.868515e+02</td>\n",
       "      <td>-0.877075</td>\n",
       "      <td>5.114658e+06</td>\n",
       "      <td>5.114764e+06</td>\n",
       "      <td>5.114868e+06</td>\n",
       "      <td>5.114876e+06</td>\n",
       "      <td>5.117490e+06</td>\n",
       "      <td>5.114873e+06</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.671143e-01</td>\n",
       "      <td>0.169159</td>\n",
       "      <td>2.860641e-02</td>\n",
       "      <td>3.796387e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.109985</td>\n",
       "      <td>2.199707e-01</td>\n",
       "      <td>1.299805e+00</td>\n",
       "      <td>0.049988</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>83.81250</td>\n",
       "      <td>90.4375</td>\n",
       "      <td>8176.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.5</td>\n",
       "      <td>140.1875</td>\n",
       "      <td>339.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.012694e+02</td>\n",
       "      <td>1.098204e+02</td>\n",
       "      <td>1.103763e+04</td>\n",
       "      <td>2.919922e-01</td>\n",
       "      <td>1.011790e+02</td>\n",
       "      <td>1.013784e+02</td>\n",
       "      <td>0.010750</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4499.50000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>63671.00000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>6.246524e+06</td>\n",
       "      <td>2255.652466</td>\n",
       "      <td>5.087968e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.246218e+06</td>\n",
       "      <td>6.246424e+06</td>\n",
       "      <td>6.246524e+06</td>\n",
       "      <td>6246528.50</td>\n",
       "      <td>6.246727e+06</td>\n",
       "      <td>6246427.500</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>5.229676e+06</td>\n",
       "      <td>1910.864990</td>\n",
       "      <td>3.651405e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.218768e+06</td>\n",
       "      <td>5.225190e+06</td>\n",
       "      <td>5.232292e+06</td>\n",
       "      <td>5.240716e+06</td>\n",
       "      <td>5.241042e+06</td>\n",
       "      <td>5.238738e+06</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>1.350586e+00</td>\n",
       "      <td>1.100586</td>\n",
       "      <td>1.209961e+00</td>\n",
       "      <td>1.311523e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.109985</td>\n",
       "      <td>0.219971</td>\n",
       "      <td>1.030273e+00</td>\n",
       "      <td>7.390625e+00</td>\n",
       "      <td>0.165039</td>\n",
       "      <td>49.000000</td>\n",
       "      <td>121.87500</td>\n",
       "      <td>104.5625</td>\n",
       "      <td>10928.0</td>\n",
       "      <td>0.388916</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>209.0000</td>\n",
       "      <td>353.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.798330e+03</td>\n",
       "      <td>7.665537e+03</td>\n",
       "      <td>1.068254e+08</td>\n",
       "      <td>9.165039e-01</td>\n",
       "      <td>7.095076e+03</td>\n",
       "      <td>7.081568e+03</td>\n",
       "      <td>0.245117</td>\n",
       "      <td>0.148193</td>\n",
       "      <td>0.048553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6749.25000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>65491.00000</td>\n",
       "      <td>89.000000</td>\n",
       "      <td>6.373115e+06</td>\n",
       "      <td>9293.086182</td>\n",
       "      <td>8.636144e+07</td>\n",
       "      <td>0.860840</td>\n",
       "      <td>6.363126e+06</td>\n",
       "      <td>6.365020e+06</td>\n",
       "      <td>6.373495e+06</td>\n",
       "      <td>6383431.25</td>\n",
       "      <td>6.391721e+06</td>\n",
       "      <td>6365174.250</td>\n",
       "      <td>89.000000</td>\n",
       "      <td>5.380901e+06</td>\n",
       "      <td>8894.610352</td>\n",
       "      <td>7.911410e+07</td>\n",
       "      <td>0.581543</td>\n",
       "      <td>5.370618e+06</td>\n",
       "      <td>5.377358e+06</td>\n",
       "      <td>5.380562e+06</td>\n",
       "      <td>5.387383e+06</td>\n",
       "      <td>5.393706e+06</td>\n",
       "      <td>5.381942e+06</td>\n",
       "      <td>39.000000</td>\n",
       "      <td>3.361328e+00</td>\n",
       "      <td>2.208984</td>\n",
       "      <td>4.882812e+00</td>\n",
       "      <td>3.457520e+00</td>\n",
       "      <td>0.109985</td>\n",
       "      <td>2.182617</td>\n",
       "      <td>3.289062</td>\n",
       "      <td>4.156250e+00</td>\n",
       "      <td>1.009375e+01</td>\n",
       "      <td>3.099609</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>159.50000</td>\n",
       "      <td>116.4375</td>\n",
       "      <td>13552.0</td>\n",
       "      <td>0.916992</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>61.0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>250.0000</td>\n",
       "      <td>358.000000</td>\n",
       "      <td>40.0</td>\n",
       "      <td>3.222136e+04</td>\n",
       "      <td>3.039950e+04</td>\n",
       "      <td>8.627117e+08</td>\n",
       "      <td>1.511719e+00</td>\n",
       "      <td>2.348107e+04</td>\n",
       "      <td>2.364605e+04</td>\n",
       "      <td>0.490479</td>\n",
       "      <td>0.285645</td>\n",
       "      <td>0.155396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8999.00000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>86016.00000</td>\n",
       "      <td>805.000000</td>\n",
       "      <td>7.121694e+06</td>\n",
       "      <td>509865.906250</td>\n",
       "      <td>2.599632e+11</td>\n",
       "      <td>22.015625</td>\n",
       "      <td>7.114805e+06</td>\n",
       "      <td>7.118270e+06</td>\n",
       "      <td>7.123392e+06</td>\n",
       "      <td>7129137.00</td>\n",
       "      <td>7.133786e+06</td>\n",
       "      <td>7121693.500</td>\n",
       "      <td>805.000000</td>\n",
       "      <td>6.761291e+06</td>\n",
       "      <td>680481.812500</td>\n",
       "      <td>4.630555e+11</td>\n",
       "      <td>20.312500</td>\n",
       "      <td>6.735117e+06</td>\n",
       "      <td>6.748980e+06</td>\n",
       "      <td>6.766294e+06</td>\n",
       "      <td>6.777269e+06</td>\n",
       "      <td>7.667580e+06</td>\n",
       "      <td>6.777668e+06</td>\n",
       "      <td>103.000000</td>\n",
       "      <td>1.425000e+01</td>\n",
       "      <td>28.875000</td>\n",
       "      <td>8.340000e+02</td>\n",
       "      <td>2.090625e+01</td>\n",
       "      <td>10.203125</td>\n",
       "      <td>10.203125</td>\n",
       "      <td>10.468750</td>\n",
       "      <td>1.085156e+01</td>\n",
       "      <td>1.001875e+02</td>\n",
       "      <td>10.523438</td>\n",
       "      <td>313.000000</td>\n",
       "      <td>360.00000</td>\n",
       "      <td>192.3750</td>\n",
       "      <td>37024.0</td>\n",
       "      <td>17.687500</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>360.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>360.0000</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>360.0</td>\n",
       "      <td>2.024536e+06</td>\n",
       "      <td>3.423408e+06</td>\n",
       "      <td>4.680157e+12</td>\n",
       "      <td>2.278000e+03</td>\n",
       "      <td>3.062169e+06</td>\n",
       "      <td>1.560282e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                ID    start_date    start_hour      end_date      end_hour  \\\n",
       "count  36000.00000  36000.000000  36000.000000  36000.000000  36000.000000   \n",
       "mean    4499.50000     14.740000      9.691444     14.356194     12.779500   \n",
       "std     2598.11228      8.807784      6.798751      8.605014      7.205213   \n",
       "min        0.00000      1.000000      0.000000      1.000000      0.000000   \n",
       "25%     2249.75000      7.000000      4.000000      6.000000      6.000000   \n",
       "50%     4499.50000     14.000000     11.000000     13.000000     11.000000   \n",
       "75%     6749.25000     21.000000     17.000000     20.000000     18.000000   \n",
       "max     8999.00000     31.000000     23.000000     31.000000     23.000000   \n",
       "\n",
       "          work_days  work_seconds     nunique_x        mean_x          std_x  \\\n",
       "count  36000.000000   36000.00000  36000.000000  3.600000e+04   36000.000000   \n",
       "mean       0.004417   62388.20900     43.585139  6.276861e+06    6881.452148   \n",
       "std        0.070377    7593.57723     43.507517  2.686800e+05   12175.655273   \n",
       "min        0.000000      90.00000      1.000000  5.135336e+06       0.000000   \n",
       "25%        0.000000   61262.75000      3.000000  6.118429e+06      23.511981   \n",
       "50%        0.000000   63671.00000     30.000000  6.246524e+06    2255.652466   \n",
       "75%        0.000000   65491.00000     89.000000  6.373115e+06    9293.086182   \n",
       "max        2.000000   86016.00000    805.000000  7.121694e+06  509865.906250   \n",
       "\n",
       "              var_x        skew_x         min_x  quantile0.25_x      median_x  \\\n",
       "count  3.600000e+04  36000.000000  3.600000e+04    3.600000e+04  3.600000e+04   \n",
       "mean   1.955948e+08      0.157837  6.265251e+06    6.271536e+06  6.276682e+06   \n",
       "std    2.279927e+09      3.158203  2.690936e+05    2.685254e+05  2.689069e+05   \n",
       "min    0.000000e+00    -16.062500  5.000250e+06    5.127822e+06  5.130528e+06   \n",
       "25%    5.528132e+02     -0.591309  6.115724e+06    6.117350e+06  6.118354e+06   \n",
       "50%    5.087968e+06      0.000000  6.246218e+06    6.246424e+06  6.246524e+06   \n",
       "75%    8.636144e+07      0.860840  6.363126e+06    6.365020e+06  6.373495e+06   \n",
       "max    2.599632e+11     22.015625  7.114805e+06    7.118270e+06  7.123392e+06   \n",
       "\n",
       "       quantile0.75_x         max_x       mode_x     nunique_y        mean_y  \\\n",
       "count        36000.00  3.600000e+04    36000.000  36000.000000  3.600000e+04   \n",
       "mean       6281997.50  6.288414e+06  6274504.000     43.585389  5.271188e+06   \n",
       "std         269354.75  2.696467e+05   268801.125     43.507534  2.548936e+05   \n",
       "min        5137676.00  5.154938e+06  5144128.000      1.000000  4.502637e+06   \n",
       "25%        6124759.25  6.134904e+06  6118252.000      3.000000  5.114772e+06   \n",
       "50%        6246528.50  6.246727e+06  6246427.500     30.000000  5.229676e+06   \n",
       "75%        6383431.25  6.391721e+06  6365174.250     89.000000  5.380901e+06   \n",
       "max        7129137.00  7.133786e+06  7121693.500    805.000000  6.761291e+06   \n",
       "\n",
       "               std_y         var_y        skew_y         min_y  \\\n",
       "count   36000.000000  3.600000e+04  36000.000000  3.600000e+04   \n",
       "mean     6120.426270  1.724055e+08     -0.187134  5.260275e+06   \n",
       "std     11616.648438  3.866452e+09      3.271484  2.554044e+05   \n",
       "min         0.000000  0.000000e+00    -28.765625  3.345433e+06   \n",
       "25%        19.668538  3.868515e+02     -0.877075  5.114658e+06   \n",
       "50%      1910.864990  3.651405e+06      0.000000  5.218768e+06   \n",
       "75%      8894.610352  7.911410e+07      0.581543  5.370618e+06   \n",
       "max    680481.812500  4.630555e+11     20.312500  6.735117e+06   \n",
       "\n",
       "       quantile0.25_y      median_y  quantile0.75_y         max_y  \\\n",
       "count    3.600000e+04  3.600000e+04    3.600000e+04  3.600000e+04   \n",
       "mean     5.266671e+06  5.271421e+06    5.275844e+06  5.280934e+06   \n",
       "std      2.547385e+05  2.550029e+05    2.554104e+05  2.570212e+05   \n",
       "min      4.469721e+06  4.496677e+06    4.512800e+06  4.543204e+06   \n",
       "25%      5.114764e+06  5.114868e+06    5.114876e+06  5.117490e+06   \n",
       "50%      5.225190e+06  5.232292e+06    5.240716e+06  5.241042e+06   \n",
       "75%      5.377358e+06  5.380562e+06    5.387383e+06  5.393706e+06   \n",
       "max      6.748980e+06  6.766294e+06    6.777269e+06  7.667580e+06   \n",
       "\n",
       "             mode_y  nunique_speed    mean_speed     std_speed     var_speed  \\\n",
       "count  3.600000e+04   36000.000000  3.600000e+04  36000.000000  3.600000e+04   \n",
       "mean   5.273068e+06      22.717139           inf      1.344727           inf   \n",
       "std    2.542882e+05      17.311172  1.762695e+00      1.200195           inf   \n",
       "min    4.505082e+06       1.000000  0.000000e+00      0.000000  0.000000e+00   \n",
       "25%    5.114873e+06       5.000000  1.671143e-01      0.169159  2.860641e-02   \n",
       "50%    5.238738e+06      22.000000  1.350586e+00      1.100586  1.209961e+00   \n",
       "75%    5.381942e+06      39.000000  3.361328e+00      2.208984  4.882812e+00   \n",
       "max    6.777668e+06     103.000000  1.425000e+01     28.875000  8.340000e+02   \n",
       "\n",
       "         skew_speed     min_speed  quantile0.25_speed  median_speed  \\\n",
       "count  3.600000e+04  36000.000000        36000.000000  36000.000000   \n",
       "mean            inf      0.147095            0.990234      1.559570   \n",
       "std    2.636719e+00      0.448975            1.388672      1.858398   \n",
       "min   -7.000000e+00      0.000000            0.000000      0.000000   \n",
       "25%    3.796387e-01      0.000000            0.000000      0.109985   \n",
       "50%    1.311523e+00      0.000000            0.109985      0.219971   \n",
       "75%    3.457520e+00      0.109985            2.182617      3.289062   \n",
       "max    2.090625e+01     10.203125           10.203125     10.468750   \n",
       "\n",
       "       quantile0.75_speed     max_speed    mode_speed  nunique_direction  \\\n",
       "count        3.600000e+04  3.600000e+04  36000.000000       36000.000000   \n",
       "mean                  inf           inf      1.490234          45.742028   \n",
       "std          2.539062e+00  5.179688e+00      2.300781          22.710759   \n",
       "min          0.000000e+00  0.000000e+00      0.000000           1.000000   \n",
       "25%          2.199707e-01  1.299805e+00      0.049988          36.000000   \n",
       "50%          1.030273e+00  7.390625e+00      0.165039          49.000000   \n",
       "75%          4.156250e+00  1.009375e+01      3.099609          61.000000   \n",
       "max          1.085156e+01  1.001875e+02     10.523438         313.000000   \n",
       "\n",
       "       mean_direction  std_direction  var_direction  skew_direction  \\\n",
       "count     36000.00000     36000.0000        36000.0    36000.000000   \n",
       "mean              inf            inf            inf        0.718262   \n",
       "std          61.09375        36.0000            inf        1.740234   \n",
       "min           0.00000         0.0000            0.0       -7.964844   \n",
       "25%          83.81250        90.4375         8176.0        0.000000   \n",
       "50%         121.87500       104.5625        10928.0        0.388916   \n",
       "75%         159.50000       116.4375        13552.0        0.916992   \n",
       "max         360.00000       192.3750        37024.0       17.687500   \n",
       "\n",
       "       min_direction  quantile0.25_direction  median_direction  \\\n",
       "count   36000.000000                 36000.0           36000.0   \n",
       "mean        4.291222                     inf               inf   \n",
       "std        27.880334                     inf               inf   \n",
       "min         0.000000                     0.0               0.0   \n",
       "25%         0.000000                     0.0               7.5   \n",
       "50%         0.000000                     5.0              91.0   \n",
       "75%         0.000000                    61.0             171.0   \n",
       "max       360.000000                   360.0             360.0   \n",
       "\n",
       "       quantile0.75_direction  max_direction  mode_direction     x_max-min  \\\n",
       "count              36000.0000   36000.000000         36000.0  3.600000e+04   \n",
       "mean                      inf     316.098194             inf  2.301732e+04   \n",
       "std                   90.8125      98.321777             inf  4.657031e+04   \n",
       "min                    0.0000       0.000000             0.0  0.000000e+00   \n",
       "25%                  140.1875     339.000000             0.0  1.012694e+02   \n",
       "50%                  209.0000     353.000000             0.0  8.798330e+03   \n",
       "75%                  250.0000     358.000000            40.0  3.222136e+04   \n",
       "max                  360.0000     360.000000           360.0  2.024536e+06   \n",
       "\n",
       "          y_max-min      rec_area         slope       short_r        long_r  \\\n",
       "count  3.600000e+04  3.600000e+04  3.600000e+04  3.600000e+04  3.600000e+04   \n",
       "mean   2.066825e+04  2.118399e+09           inf  1.791151e+04  1.693063e+04   \n",
       "std    4.740188e+04  4.797043e+10           inf  5.023682e+04  3.199636e+04   \n",
       "min    0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "25%    1.098204e+02  1.103763e+04  2.919922e-01  1.011790e+02  1.013784e+02   \n",
       "50%    7.665537e+03  1.068254e+08  9.165039e-01  7.095076e+03  7.081568e+03   \n",
       "75%    3.039950e+04  8.627117e+08  1.511719e+00  2.348107e+04  2.364605e+04   \n",
       "max    3.423408e+06  4.680157e+12  2.278000e+03  3.062169e+06  1.560282e+06   \n",
       "\n",
       "       direction_miss_rate  speed_miss_rate  direct&speed_miss_rate  \n",
       "count         36000.000000     36000.000000            36000.000000  \n",
       "mean              0.311768         0.196655                0.139160  \n",
       "std               0.321533         0.228027                0.225464  \n",
       "min               0.000000         0.000000                0.000000  \n",
       "25%               0.010750         0.000000                0.000000  \n",
       "50%               0.245117         0.148193                0.048553  \n",
       "75%               0.490479         0.285645                0.155396  \n",
       "max               1.000000         1.000000                1.000000  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()\n",
    "#mode_direction, "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ID', 'start_date', 'start_hour', 'end_date', 'end_hour', 'work_days',\n",
       "       'work_seconds', 'nunique_x', 'mean_x', 'std_x', 'var_x', 'skew_x',\n",
       "       'min_x', 'quantile0.25_x', 'median_x', 'quantile0.75_x', 'max_x',\n",
       "       'mode_x', 'nunique_y', 'mean_y', 'std_y', 'var_y', 'skew_y', 'min_y',\n",
       "       'quantile0.25_y', 'median_y', 'quantile0.75_y', 'max_y', 'mode_y',\n",
       "       'nunique_speed', 'mean_speed', 'std_speed', 'var_speed', 'skew_speed',\n",
       "       'min_speed', 'quantile0.25_speed', 'median_speed', 'quantile0.75_speed',\n",
       "       'max_speed', 'mode_speed', 'nunique_direction', 'mean_direction',\n",
       "       'std_direction', 'var_direction', 'skew_direction', 'min_direction',\n",
       "       'quantile0.25_direction', 'median_direction', 'quantile0.75_direction',\n",
       "       'max_direction', 'mode_direction', 'x_max-min', 'y_max-min', 'rec_area',\n",
       "       'slope', 'short_r', 'long_r', 'direction_miss_rate', 'speed_miss_rate',\n",
       "       'direct&speed_miss_rate', 'type'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#所有特征\n",
    "data.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#设置去掉的特征\n",
    "NotimportantFeats = ['end_date', 'start_date', 'start_hour', 'end_hour', 'work_days', 'work_seconds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#分离训练集和测试集\n",
    "train = data[data.type!=\"测试\"]\n",
    "test = data[data.type==\"测试\"]\n",
    "\n",
    "#特征选择,X,y分离\n",
    "train_x = train[[i for i in train.columns if i not in ['ID', 'time', 'type']+NotimportantFeats]]\n",
    "test_x = test[[i for i in test.columns if i not in ['ID', 'time', 'type']+NotimportantFeats]]\n",
    "\n",
    "train_y = train['type']\n",
    "\n",
    "#label和type互相装化\n",
    "label2type = dict(zip(range(0, len(set(train_y))), sorted(list(set(train_y)))))\n",
    "type2label = dict(zip(sorted(list(set(train_y))), range(0, len(set(train_y)))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_pred = []\n",
    "oof = train[['ID']]\n",
    "skf = StratifiedKFold(n_splits=5, random_state=27, shuffle=True)\n",
    "# skf = KFold(n_splits=20, shuffle=True, random_state=27)\n",
    "\n",
    "feature_importances = pd.DataFrame()\n",
    "feature_importances['feature'] = train_x.columns\n",
    "\n",
    "for index, (train_index, val_index) in enumerate(skf.split(train_x, train_y)):\n",
    "    \n",
    "    model = lgb.LGBMClassifier(\n",
    "        boosting_type=\"gbdt\", num_leaves=120, reg_alpha=0, reg_lambda=0.,\n",
    "        max_depth=-1, n_estimators=800, objective='multiclass', class_weight='balanced',\n",
    "        subsample=0.9, colsample_bytree=0.5, subsample_freq=1,\n",
    "        learning_rate=0.03, random_state=2018 + index, n_jobs=10, metric=\"None\", importance_type='gain'\n",
    "    )\n",
    "    \n",
    "    train_x1, val_x1, train_y1, val_y1 = \\\n",
    "    train_x.loc[train_index], train_x.loc[val_index], train_y.loc[train_index], train_y.loc[val_index]\n",
    "\n",
    "    model.fit(train_x1, train_y1)\n",
    "    \n",
    "    #out of folder预测\n",
    "    oof.loc[val_index] = model.predict(val_x1).reshape(-1, 1)\n",
    "    \n",
    "    #测试集预测\n",
    "    test_y = model.predict(test_x)\n",
    "    test_y = pd.Series(test_y).map(type2label)\n",
    "    \n",
    "    #特征重要性\n",
    "    feature_importances['fold_{}'.format(index + 1)] = model.feature_importances_\n",
    "    \n",
    "    if index == 0:\n",
    "        cv_pred = np.array(test_y).reshape(-1, 1)\n",
    "    else:\n",
    "        cv_pred = np.hstack((cv_pred, np.array(test_y).reshape(-1, 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#投票策略筛选预测结果\n",
    "y_pred = []\n",
    "for line in np.array(oof['ID'].map(type2label)).reshape(-1,4):\n",
    "    y_pred.append(np.argmax(np.bincount(line)))\n",
    "\n",
    "#投票策略筛选预测结果\n",
    "y_true = []\n",
    "for line in np.array(train['type'].map(type2label)).reshape(-1,4):\n",
    "    y_true.append(np.argmax(np.bincount(line)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9403440104148132"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#oof F1-score\n",
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_true=y_true, y_pred=y_pred, average='macro')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征重要性分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>fold_1</th>\n",
       "      <th>fold_2</th>\n",
       "      <th>fold_3</th>\n",
       "      <th>fold_4</th>\n",
       "      <th>fold_5</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>mode_y</td>\n",
       "      <td>65140.492572</td>\n",
       "      <td>53488.887097</td>\n",
       "      <td>77436.688116</td>\n",
       "      <td>53535.256732</td>\n",
       "      <td>54231.462980</td>\n",
       "      <td>303832.787496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>quantile0.75_y</td>\n",
       "      <td>50313.786235</td>\n",
       "      <td>55054.687397</td>\n",
       "      <td>43327.517186</td>\n",
       "      <td>67617.475788</td>\n",
       "      <td>66608.435371</td>\n",
       "      <td>282921.901977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>min_y</td>\n",
       "      <td>50600.354792</td>\n",
       "      <td>51092.671145</td>\n",
       "      <td>40862.560985</td>\n",
       "      <td>42094.840140</td>\n",
       "      <td>47068.011040</td>\n",
       "      <td>231718.438103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>max_y</td>\n",
       "      <td>42758.987907</td>\n",
       "      <td>49641.179387</td>\n",
       "      <td>39324.248159</td>\n",
       "      <td>39300.098361</td>\n",
       "      <td>47690.840814</td>\n",
       "      <td>218715.354627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>quantile0.25_y</td>\n",
       "      <td>39803.015243</td>\n",
       "      <td>40831.414000</td>\n",
       "      <td>44586.028817</td>\n",
       "      <td>40935.076539</td>\n",
       "      <td>34619.052113</td>\n",
       "      <td>200774.586712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>median_speed</td>\n",
       "      <td>43888.840835</td>\n",
       "      <td>39492.659553</td>\n",
       "      <td>46297.548634</td>\n",
       "      <td>33963.410991</td>\n",
       "      <td>34286.748254</td>\n",
       "      <td>197929.208267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mean_y</td>\n",
       "      <td>38745.978898</td>\n",
       "      <td>44470.218137</td>\n",
       "      <td>33652.137937</td>\n",
       "      <td>40149.769091</td>\n",
       "      <td>35420.307149</td>\n",
       "      <td>192438.411212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>median_y</td>\n",
       "      <td>35639.698002</td>\n",
       "      <td>27755.696246</td>\n",
       "      <td>42679.354475</td>\n",
       "      <td>39464.773718</td>\n",
       "      <td>38293.879754</td>\n",
       "      <td>183833.402194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>mode_x</td>\n",
       "      <td>34063.971810</td>\n",
       "      <td>31286.317083</td>\n",
       "      <td>31862.951598</td>\n",
       "      <td>28430.354792</td>\n",
       "      <td>30995.664159</td>\n",
       "      <td>156639.259442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>quantile0.25_x</td>\n",
       "      <td>22291.897050</td>\n",
       "      <td>24923.249905</td>\n",
       "      <td>29933.335720</td>\n",
       "      <td>34002.606460</td>\n",
       "      <td>27141.391789</td>\n",
       "      <td>138292.480922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>median_x</td>\n",
       "      <td>22505.386388</td>\n",
       "      <td>26729.066998</td>\n",
       "      <td>25692.087625</td>\n",
       "      <td>20545.270472</td>\n",
       "      <td>24666.515091</td>\n",
       "      <td>120138.326574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>min_x</td>\n",
       "      <td>22906.684595</td>\n",
       "      <td>25573.928452</td>\n",
       "      <td>19490.311952</td>\n",
       "      <td>25785.289217</td>\n",
       "      <td>24118.921791</td>\n",
       "      <td>117875.136006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>quantile0.75_x</td>\n",
       "      <td>25085.362476</td>\n",
       "      <td>22269.668748</td>\n",
       "      <td>27202.280730</td>\n",
       "      <td>21329.443682</td>\n",
       "      <td>20224.450649</td>\n",
       "      <td>116111.206285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max_x</td>\n",
       "      <td>20964.350921</td>\n",
       "      <td>21436.244613</td>\n",
       "      <td>20209.442649</td>\n",
       "      <td>20859.070836</td>\n",
       "      <td>20210.745387</td>\n",
       "      <td>103679.854405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mean_x</td>\n",
       "      <td>19993.180926</td>\n",
       "      <td>18412.696853</td>\n",
       "      <td>14614.245745</td>\n",
       "      <td>19094.942738</td>\n",
       "      <td>17757.175420</td>\n",
       "      <td>89872.241682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>rec_area</td>\n",
       "      <td>16049.553051</td>\n",
       "      <td>9482.911397</td>\n",
       "      <td>12927.734290</td>\n",
       "      <td>19421.653804</td>\n",
       "      <td>19583.997126</td>\n",
       "      <td>77465.849668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>quantile0.25_speed</td>\n",
       "      <td>12364.249781</td>\n",
       "      <td>12386.155578</td>\n",
       "      <td>13651.939195</td>\n",
       "      <td>14145.674927</td>\n",
       "      <td>13645.975231</td>\n",
       "      <td>66193.994712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>mean_speed</td>\n",
       "      <td>11346.990755</td>\n",
       "      <td>18387.502023</td>\n",
       "      <td>12654.709744</td>\n",
       "      <td>10805.676819</td>\n",
       "      <td>12059.790726</td>\n",
       "      <td>65254.670067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>quantile0.75_speed</td>\n",
       "      <td>10241.350835</td>\n",
       "      <td>10475.178678</td>\n",
       "      <td>9405.562789</td>\n",
       "      <td>16063.404733</td>\n",
       "      <td>15892.144964</td>\n",
       "      <td>62077.641999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>direction_miss_rate</td>\n",
       "      <td>9534.288189</td>\n",
       "      <td>9175.638005</td>\n",
       "      <td>10848.014340</td>\n",
       "      <td>9732.063057</td>\n",
       "      <td>9883.138642</td>\n",
       "      <td>49173.142233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>nunique_direction</td>\n",
       "      <td>8316.220161</td>\n",
       "      <td>9178.375703</td>\n",
       "      <td>8868.723303</td>\n",
       "      <td>9036.541328</td>\n",
       "      <td>10083.123802</td>\n",
       "      <td>45482.984297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>mode_speed</td>\n",
       "      <td>8467.451566</td>\n",
       "      <td>8337.684218</td>\n",
       "      <td>7926.285713</td>\n",
       "      <td>7516.699817</td>\n",
       "      <td>7416.812124</td>\n",
       "      <td>39664.933437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>std_speed</td>\n",
       "      <td>8970.459523</td>\n",
       "      <td>7218.568495</td>\n",
       "      <td>7000.580124</td>\n",
       "      <td>7044.749926</td>\n",
       "      <td>8334.363457</td>\n",
       "      <td>38568.721525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>speed_miss_rate</td>\n",
       "      <td>6838.424526</td>\n",
       "      <td>8484.357212</td>\n",
       "      <td>7624.646384</td>\n",
       "      <td>7047.402349</td>\n",
       "      <td>7816.500901</td>\n",
       "      <td>37811.331372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>long_r</td>\n",
       "      <td>8000.872235</td>\n",
       "      <td>7872.889870</td>\n",
       "      <td>8350.023194</td>\n",
       "      <td>6745.428933</td>\n",
       "      <td>6694.777766</td>\n",
       "      <td>37663.991998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>std_x</td>\n",
       "      <td>6451.167269</td>\n",
       "      <td>6332.490123</td>\n",
       "      <td>7619.306094</td>\n",
       "      <td>7708.278398</td>\n",
       "      <td>6577.687345</td>\n",
       "      <td>34688.929230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>direct&amp;speed_miss_rate</td>\n",
       "      <td>6534.325416</td>\n",
       "      <td>6390.471217</td>\n",
       "      <td>6836.037853</td>\n",
       "      <td>6515.358697</td>\n",
       "      <td>6045.425340</td>\n",
       "      <td>32321.618523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>x_max-min</td>\n",
       "      <td>6753.693444</td>\n",
       "      <td>6041.360443</td>\n",
       "      <td>6429.895406</td>\n",
       "      <td>6339.939841</td>\n",
       "      <td>6338.438100</td>\n",
       "      <td>31903.327236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>skew_y</td>\n",
       "      <td>5646.524024</td>\n",
       "      <td>5865.991038</td>\n",
       "      <td>6408.427733</td>\n",
       "      <td>6265.085736</td>\n",
       "      <td>6341.528304</td>\n",
       "      <td>30527.556834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>skew_x</td>\n",
       "      <td>6053.381732</td>\n",
       "      <td>6386.467989</td>\n",
       "      <td>5994.781185</td>\n",
       "      <td>5441.541972</td>\n",
       "      <td>6129.585109</td>\n",
       "      <td>30005.757987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>skew_speed</td>\n",
       "      <td>5807.646795</td>\n",
       "      <td>5772.967929</td>\n",
       "      <td>5568.749977</td>\n",
       "      <td>6294.954137</td>\n",
       "      <td>5999.367318</td>\n",
       "      <td>29443.686157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nunique_x</td>\n",
       "      <td>5632.918524</td>\n",
       "      <td>5940.092926</td>\n",
       "      <td>5497.534612</td>\n",
       "      <td>5442.521798</td>\n",
       "      <td>6349.881993</td>\n",
       "      <td>28862.949852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>mean_direction</td>\n",
       "      <td>5237.349561</td>\n",
       "      <td>5538.339977</td>\n",
       "      <td>5917.172871</td>\n",
       "      <td>5815.454081</td>\n",
       "      <td>6231.930788</td>\n",
       "      <td>28740.247279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>short_r</td>\n",
       "      <td>6376.931139</td>\n",
       "      <td>6341.779873</td>\n",
       "      <td>5449.873517</td>\n",
       "      <td>5149.396983</td>\n",
       "      <td>5334.235534</td>\n",
       "      <td>28652.217046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>nunique_speed</td>\n",
       "      <td>5467.052448</td>\n",
       "      <td>5675.955579</td>\n",
       "      <td>5403.787032</td>\n",
       "      <td>5130.824216</td>\n",
       "      <td>6094.441954</td>\n",
       "      <td>27772.061229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>std_y</td>\n",
       "      <td>5522.965265</td>\n",
       "      <td>4948.669263</td>\n",
       "      <td>5457.850219</td>\n",
       "      <td>5586.028512</td>\n",
       "      <td>5073.937152</td>\n",
       "      <td>26589.450412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>quantile0.75_direction</td>\n",
       "      <td>5007.096665</td>\n",
       "      <td>5382.525779</td>\n",
       "      <td>5455.584335</td>\n",
       "      <td>5399.974993</td>\n",
       "      <td>4917.150269</td>\n",
       "      <td>26162.332040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>var_speed</td>\n",
       "      <td>4585.281713</td>\n",
       "      <td>5286.911711</td>\n",
       "      <td>5111.682557</td>\n",
       "      <td>5467.868586</td>\n",
       "      <td>5451.559406</td>\n",
       "      <td>25903.303974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>max_speed</td>\n",
       "      <td>5206.227470</td>\n",
       "      <td>5290.715028</td>\n",
       "      <td>4620.032395</td>\n",
       "      <td>4847.668720</td>\n",
       "      <td>4601.677631</td>\n",
       "      <td>24566.321243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>slope</td>\n",
       "      <td>5058.744037</td>\n",
       "      <td>5001.409259</td>\n",
       "      <td>4233.060652</td>\n",
       "      <td>5185.788201</td>\n",
       "      <td>4871.829121</td>\n",
       "      <td>24350.831270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>std_direction</td>\n",
       "      <td>4531.043707</td>\n",
       "      <td>5148.624883</td>\n",
       "      <td>4207.568199</td>\n",
       "      <td>4165.836609</td>\n",
       "      <td>4528.437740</td>\n",
       "      <td>22581.511138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>skew_direction</td>\n",
       "      <td>4256.630524</td>\n",
       "      <td>4759.795356</td>\n",
       "      <td>4490.398371</td>\n",
       "      <td>4523.619103</td>\n",
       "      <td>4059.763465</td>\n",
       "      <td>22090.206819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>y_max-min</td>\n",
       "      <td>3912.949288</td>\n",
       "      <td>3968.166480</td>\n",
       "      <td>4336.614567</td>\n",
       "      <td>4279.682288</td>\n",
       "      <td>4277.436038</td>\n",
       "      <td>20774.848660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>median_direction</td>\n",
       "      <td>3799.721849</td>\n",
       "      <td>3973.523451</td>\n",
       "      <td>3394.771122</td>\n",
       "      <td>3813.672817</td>\n",
       "      <td>3664.363664</td>\n",
       "      <td>18646.052903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>max_direction</td>\n",
       "      <td>3314.235704</td>\n",
       "      <td>3330.446407</td>\n",
       "      <td>3260.166834</td>\n",
       "      <td>3518.959161</td>\n",
       "      <td>3545.991479</td>\n",
       "      <td>16969.799584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>var_x</td>\n",
       "      <td>3488.414007</td>\n",
       "      <td>3457.598229</td>\n",
       "      <td>2644.819758</td>\n",
       "      <td>3339.753732</td>\n",
       "      <td>3243.243501</td>\n",
       "      <td>16173.829227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>var_y</td>\n",
       "      <td>3199.073049</td>\n",
       "      <td>3756.966185</td>\n",
       "      <td>3382.720472</td>\n",
       "      <td>2779.593303</td>\n",
       "      <td>2927.715916</td>\n",
       "      <td>16046.068926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>var_direction</td>\n",
       "      <td>3149.066802</td>\n",
       "      <td>2912.769017</td>\n",
       "      <td>2735.272574</td>\n",
       "      <td>3036.830293</td>\n",
       "      <td>2825.702373</td>\n",
       "      <td>14659.641059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>nunique_y</td>\n",
       "      <td>3043.639505</td>\n",
       "      <td>2897.608830</td>\n",
       "      <td>2690.408054</td>\n",
       "      <td>2648.411171</td>\n",
       "      <td>3297.509485</td>\n",
       "      <td>14577.577046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>quantile0.25_direction</td>\n",
       "      <td>3256.139165</td>\n",
       "      <td>2732.000903</td>\n",
       "      <td>2653.586326</td>\n",
       "      <td>2771.250367</td>\n",
       "      <td>3134.029730</td>\n",
       "      <td>14547.006491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>mode_direction</td>\n",
       "      <td>2695.717411</td>\n",
       "      <td>2761.403565</td>\n",
       "      <td>2705.116381</td>\n",
       "      <td>2770.074127</td>\n",
       "      <td>2598.168138</td>\n",
       "      <td>13530.479622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>min_speed</td>\n",
       "      <td>2505.398512</td>\n",
       "      <td>2107.058380</td>\n",
       "      <td>2279.818520</td>\n",
       "      <td>2668.960281</td>\n",
       "      <td>2346.828889</td>\n",
       "      <td>11908.064583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>min_direction</td>\n",
       "      <td>475.607496</td>\n",
       "      <td>428.908747</td>\n",
       "      <td>429.784888</td>\n",
       "      <td>341.017410</td>\n",
       "      <td>434.020647</td>\n",
       "      <td>2109.339188</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   feature        fold_1        fold_2        fold_3  \\\n",
       "21                  mode_y  65140.492572  53488.887097  77436.688116   \n",
       "19          quantile0.75_y  50313.786235  55054.687397  43327.517186   \n",
       "16                   min_y  50600.354792  51092.671145  40862.560985   \n",
       "20                   max_y  42758.987907  49641.179387  39324.248159   \n",
       "17          quantile0.25_y  39803.015243  40831.414000  44586.028817   \n",
       "29            median_speed  43888.840835  39492.659553  46297.548634   \n",
       "12                  mean_y  38745.978898  44470.218137  33652.137937   \n",
       "18                median_y  35639.698002  27755.696246  42679.354475   \n",
       "10                  mode_x  34063.971810  31286.317083  31862.951598   \n",
       "6           quantile0.25_x  22291.897050  24923.249905  29933.335720   \n",
       "7                 median_x  22505.386388  26729.066998  25692.087625   \n",
       "5                    min_x  22906.684595  25573.928452  19490.311952   \n",
       "8           quantile0.75_x  25085.362476  22269.668748  27202.280730   \n",
       "9                    max_x  20964.350921  21436.244613  20209.442649   \n",
       "1                   mean_x  19993.180926  18412.696853  14614.245745   \n",
       "46                rec_area  16049.553051   9482.911397  12927.734290   \n",
       "28      quantile0.25_speed  12364.249781  12386.155578  13651.939195   \n",
       "23              mean_speed  11346.990755  18387.502023  12654.709744   \n",
       "30      quantile0.75_speed  10241.350835  10475.178678   9405.562789   \n",
       "50     direction_miss_rate   9534.288189   9175.638005  10848.014340   \n",
       "33       nunique_direction   8316.220161   9178.375703   8868.723303   \n",
       "32              mode_speed   8467.451566   8337.684218   7926.285713   \n",
       "24               std_speed   8970.459523   7218.568495   7000.580124   \n",
       "51         speed_miss_rate   6838.424526   8484.357212   7624.646384   \n",
       "49                  long_r   8000.872235   7872.889870   8350.023194   \n",
       "2                    std_x   6451.167269   6332.490123   7619.306094   \n",
       "52  direct&speed_miss_rate   6534.325416   6390.471217   6836.037853   \n",
       "44               x_max-min   6753.693444   6041.360443   6429.895406   \n",
       "15                  skew_y   5646.524024   5865.991038   6408.427733   \n",
       "4                   skew_x   6053.381732   6386.467989   5994.781185   \n",
       "26              skew_speed   5807.646795   5772.967929   5568.749977   \n",
       "0                nunique_x   5632.918524   5940.092926   5497.534612   \n",
       "34          mean_direction   5237.349561   5538.339977   5917.172871   \n",
       "48                 short_r   6376.931139   6341.779873   5449.873517   \n",
       "22           nunique_speed   5467.052448   5675.955579   5403.787032   \n",
       "13                   std_y   5522.965265   4948.669263   5457.850219   \n",
       "41  quantile0.75_direction   5007.096665   5382.525779   5455.584335   \n",
       "25               var_speed   4585.281713   5286.911711   5111.682557   \n",
       "31               max_speed   5206.227470   5290.715028   4620.032395   \n",
       "47                   slope   5058.744037   5001.409259   4233.060652   \n",
       "35           std_direction   4531.043707   5148.624883   4207.568199   \n",
       "37          skew_direction   4256.630524   4759.795356   4490.398371   \n",
       "45               y_max-min   3912.949288   3968.166480   4336.614567   \n",
       "40        median_direction   3799.721849   3973.523451   3394.771122   \n",
       "42           max_direction   3314.235704   3330.446407   3260.166834   \n",
       "3                    var_x   3488.414007   3457.598229   2644.819758   \n",
       "14                   var_y   3199.073049   3756.966185   3382.720472   \n",
       "36           var_direction   3149.066802   2912.769017   2735.272574   \n",
       "11               nunique_y   3043.639505   2897.608830   2690.408054   \n",
       "39  quantile0.25_direction   3256.139165   2732.000903   2653.586326   \n",
       "43          mode_direction   2695.717411   2761.403565   2705.116381   \n",
       "27               min_speed   2505.398512   2107.058380   2279.818520   \n",
       "38           min_direction    475.607496    428.908747    429.784888   \n",
       "\n",
       "          fold_4        fold_5     importance  \n",
       "21  53535.256732  54231.462980  303832.787496  \n",
       "19  67617.475788  66608.435371  282921.901977  \n",
       "16  42094.840140  47068.011040  231718.438103  \n",
       "20  39300.098361  47690.840814  218715.354627  \n",
       "17  40935.076539  34619.052113  200774.586712  \n",
       "29  33963.410991  34286.748254  197929.208267  \n",
       "12  40149.769091  35420.307149  192438.411212  \n",
       "18  39464.773718  38293.879754  183833.402194  \n",
       "10  28430.354792  30995.664159  156639.259442  \n",
       "6   34002.606460  27141.391789  138292.480922  \n",
       "7   20545.270472  24666.515091  120138.326574  \n",
       "5   25785.289217  24118.921791  117875.136006  \n",
       "8   21329.443682  20224.450649  116111.206285  \n",
       "9   20859.070836  20210.745387  103679.854405  \n",
       "1   19094.942738  17757.175420   89872.241682  \n",
       "46  19421.653804  19583.997126   77465.849668  \n",
       "28  14145.674927  13645.975231   66193.994712  \n",
       "23  10805.676819  12059.790726   65254.670067  \n",
       "30  16063.404733  15892.144964   62077.641999  \n",
       "50   9732.063057   9883.138642   49173.142233  \n",
       "33   9036.541328  10083.123802   45482.984297  \n",
       "32   7516.699817   7416.812124   39664.933437  \n",
       "24   7044.749926   8334.363457   38568.721525  \n",
       "51   7047.402349   7816.500901   37811.331372  \n",
       "49   6745.428933   6694.777766   37663.991998  \n",
       "2    7708.278398   6577.687345   34688.929230  \n",
       "52   6515.358697   6045.425340   32321.618523  \n",
       "44   6339.939841   6338.438100   31903.327236  \n",
       "15   6265.085736   6341.528304   30527.556834  \n",
       "4    5441.541972   6129.585109   30005.757987  \n",
       "26   6294.954137   5999.367318   29443.686157  \n",
       "0    5442.521798   6349.881993   28862.949852  \n",
       "34   5815.454081   6231.930788   28740.247279  \n",
       "48   5149.396983   5334.235534   28652.217046  \n",
       "22   5130.824216   6094.441954   27772.061229  \n",
       "13   5586.028512   5073.937152   26589.450412  \n",
       "41   5399.974993   4917.150269   26162.332040  \n",
       "25   5467.868586   5451.559406   25903.303974  \n",
       "31   4847.668720   4601.677631   24566.321243  \n",
       "47   5185.788201   4871.829121   24350.831270  \n",
       "35   4165.836609   4528.437740   22581.511138  \n",
       "37   4523.619103   4059.763465   22090.206819  \n",
       "45   4279.682288   4277.436038   20774.848660  \n",
       "40   3813.672817   3664.363664   18646.052903  \n",
       "42   3518.959161   3545.991479   16969.799584  \n",
       "3    3339.753732   3243.243501   16173.829227  \n",
       "14   2779.593303   2927.715916   16046.068926  \n",
       "36   3036.830293   2825.702373   14659.641059  \n",
       "11   2648.411171   3297.509485   14577.577046  \n",
       "39   2771.250367   3134.029730   14547.006491  \n",
       "43   2770.074127   2598.168138   13530.479622  \n",
       "27   2668.960281   2346.828889   11908.064583  \n",
       "38    341.017410    434.020647    2109.339188  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_importances['importance'] = feature_importances[[i for i in feature_importances.columns if i != 'feature']].apply(lambda x: x.sum(), axis=1)\n",
    "feature_importances.sort_values(by='importance',ascending=False, inplace=True)\n",
    "feature_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['min_speed', 'min_direction']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NotimportantFeats = list(feature_importances['feature'][-2:])\n",
    "NotimportantFeats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试集预测保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2000 \n",
      " 拖网    1233\n",
      "围网     512\n",
      "刺网     255\n",
      "Name: predict, dtype: int64\n",
      "         ID predict\n",
      "35768  7000      围网\n",
      "33024  7001      拖网\n",
      "32108  7002      围网\n",
      "29832  7003      拖网\n",
      "28496  7004      围网\n"
     ]
    }
   ],
   "source": [
    "#投票策略筛选预测结果\n",
    "submit = []\n",
    "for line in cv_pred.reshape(2000,-1):\n",
    "    submit.append(np.argmax(np.bincount(line)))\n",
    "\n",
    "#预测结果\n",
    "res = test[['ID']].drop_duplicates('ID')\n",
    "res['predict'] = submit\n",
    "res['predict'] = res['predict'].map(label2type)\n",
    "\n",
    "print(len(res), '\\n',res.predict.value_counts())\n",
    "print(res.sort_values('ID').head())\n",
    "\n",
    "#保存模型\n",
    "res.sort_values('ID').to_csv('../output/'+filename+'_submission.csv', index=False, header=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Draft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feat_kernelMedian(df, df_feature, fe, value, pr, name=\"\"):\n",
    "    def get_median(a, pr=pr):\n",
    "        a = np.array(a)\n",
    "        x = a[~np.isnan(a)]\n",
    "        n = len(x)\n",
    "        weight = np.repeat(1.0, n)\n",
    "        idx = np.argsort(x)\n",
    "        x = x[idx]\n",
    "        if n<pr.shape[0]:\n",
    "            pr = pr[n,:n]\n",
    "        else:\n",
    "            scale = (n-1)/2.\n",
    "            xxx = np.arange(-(n+1)/2.+1, (n+1)/2., step=1)/scale\n",
    "            yyy = 3./4.*(1-xxx**2)\n",
    "            yyy = yyy/np.sum(yyy)\n",
    "            pr = (yyy*n+1)/(n+1)\n",
    "        ans = np.sum(pr*x*weight) / float(np.sum(pr * weight))\n",
    "        return ans\n",
    "\n",
    "    df_count = pd.DataFrame(df_feature.groupby(fe)[value].apply(get_median)).reset_index()\n",
    "    if not name:\n",
    "        df_count.columns = fe + [value+\"_%s_mean\" % (\"_\".join(fe))]\n",
    "    else:\n",
    "        df_count.columns = fe + [name]\n",
    "    df = df.merge(df_count, on=fe, how=\"left\").fillna(0)\n",
    "    return df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
